#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   domain_info_no_api.py    
@Contact :   291622538@qq.com

@Modify Time      @Author    @Version    @Description
------------      -------    --------    -----------
2020/11/8 20:59   fan        1.0         None
'''

import datetime
from scrapy import Spider
from scrapy import Request
from website.items import DomainInfoOriginalOverItem, DomainInfo


class DomainInfoGet(Spider):
    # 定义爬虫名称
    name = "domain_info_get"

    custom_settings = {
        'ITEM_PIPELINES': {'website.pipelines.SaveDomainInfoPipeline': 300},
    }

    def start_requests(self):

        with open("./data/domain_data/no_api_result.txt", "r", encoding="utf-8") as f:
            for num, domainName in enumerate(f, 1):
                if num == 20:
                    break
                domainName = domainName.replace("\n", "")
                # domainName = "360.cn"
                URL = "http://whois.world-server.net/whois.php?query=%s" % domainName
                # dont_filter 相同的url只请求一次： False
                yield Request(URL, callback=self.original_parse, meta={"domainName": domainName},
                              dont_filter=False,
                              errback=lambda response, domainName=domainName: self.save_overtime(
                                  response, domainName))

    def original_parse(self, response, **kwargs):

        try:
            text_result = response.xpath('//*[@id="registrar_whois"]/div/div/span').extract()[0]

            if "Name Server" in text_result:
                registryDomainId = ""
                domainStatus = []
                nameServers = []
                registrationDate = ""
                expirationDate = ""
                IANAID = ""
                addDate = datetime.datetime.now().strftime('%Y-%m-%d')
                registrarName = ""

                text_result = text_result.replace("<br>", "").replace("\r", "")
                for text in text_result.split("\n"):
                    text_split = text.split(":", 1)
                    if len(text_split) == 2:
                        key = text_split[0]
                        info = text_split[1].replace(" ", "").replace("</span>", "")
                        if key == "ROID" or key == "Registry Domain ID":
                            registryDomainId = info
                        elif key == "Domain Status":
                            domainStatus.append(info.split("http")[0])
                        elif key == "Name Server":
                            nameServers.append(info)
                        elif key == "Registration Time" or key == "Creation Date":
                            registrationDate = info
                        elif key == "Expiration Time" or key == "Registry Expiry Date":
                            expirationDate = info
                        elif key == "Registrar IANA ID":
                            IANAID = info
                        elif key == "Registrant" or key == "Registrar":
                            registrarName = info

                domainInfo = DomainInfo(domainName=response.meta["domainName"], registryDomainId=registryDomainId,
                                        domainStatus=";".join(domainStatus),
                                        nameServers=";".join(nameServers), registrationDate=registrationDate,
                                        expirationDate=expirationDate, IANAID=IANAID, registrarName=registrarName,
                                        addDate=addDate
                                        )
                yield domainInfo


            else:
                over_domain = DomainInfoOriginalOverItem(domainName=response.meta["domainName"])
                yield over_domain
        except Exception as e:
            print(e)
            over_domain = DomainInfoOriginalOverItem(domainName=response.meta["domainName"])
            yield over_domain

    def save_overtime(self, response, domainName):

        over_domain = DomainInfoOriginalOverItem(domainName=domainName)
        yield over_domain
